﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Text.RegularExpressions;
using System.Text;
using System.Threading.Tasks;
using System.Windows.Forms;

namespace craw
{
    internal static class Program
    {
        /// <summary>
        /// 应用程序的主入口点。
        /// </summary>
        [STAThread]
        static void Main()
        {
            string startUrl = "http://www.cnblogs.com/dstang2000/";
            if (args.Length >= 1)
                startUrl = args[0];

            Crawler myCrawler = new Crawler();
            myCrawler.StartCrawling(startUrl);
            Application.EnableVisualStyles();
            Application.SetCompatibleTextRenderingDefault(false);
            Application.Run(new Form1());
        }
    }
    public class Crawler
    {
        private HashSet<string> visitedUrls = new HashSet<string>();
        private List<string> errorUrls = new List<string>();
        private string baseUrl;

        public void StartCrawling(string initialUrl)
        {
            baseUrl = GetBaseUrl(initialUrl);
            visitedUrls.Add(baseUrl);

            Application.EnableVisualStyles();
            Application.SetCompatibleTextRenderingDefault(false);
            Application.Run(new CrawlerForm(this));
        }

        public void Crawl()
        {
            while (visitedUrls.Count > 0)
            {
                string currentUrl = GetNextUrl();
                if (currentUrl == null)
                    continue;

                try
                {
                    string htmlContent = DownloadHtml(currentUrl);
                    ParseHtmlAndExtractUrls(htmlContent);
                }
                catch (Exception ex)
                {
                    errorUrls.Add(currentUrl);
                    Console.WriteLine($"Error crawling URL: {currentUrl}. Exception: {ex.Message}");
                }
            }
        }

        private string GetNextUrl()
        {
            lock (visitedUrls)
            {
                foreach (string url in visitedUrls)
                {
                    if (!url.StartsWith("http")) // Skip non-HTTP URLs
                        continue;

                    if (!visitedUrls.Contains(url))
                    {
                        visitedUrls.Add(url);
                        return url;
                    }
                }
            }
            return null;
        }

        private string DownloadHtml(string url)
        {
            using (WebClient client = new WebClient())
            {
                client.Encoding = Encoding.UTF8;
                return client.DownloadString(url);
            }
        }

        private void ParseHtmlAndExtractUrls(string htmlContent)
        {
            // Use regular expressions or an HTML parser library to extract URLs from the HTML content
            // For demonstration purposes, let's assume we find URLs in the form of <a href="...">

            string pattern = @"<a\s+href\s*=\s*""([^""]+)""";
            MatchCollection matches = Regex.Matches(htmlContent, pattern);

            foreach (Match match in matches)
            {
                string relativeUrl = match.Groups[1].Value;
                string absoluteUrl = ConvertToAbsoluteUrl(relativeUrl);

                if (!visitedUrls.Contains(absoluteUrl))
                    visitedUrls.Add(absoluteUrl);
            }
        }

        private string ConvertToAbsoluteUrl(string relativeUrl)
        {
            if (relativeUrl.StartsWith("/"))
                return baseUrl + relativeUrl;
            else
                return relativeUrl;
        }

        private string GetBaseUrl(string url)
        {
            Uri uri = new Uri(url);
            return $"{uri.Scheme}://{uri.Host}";
        }

        public HashSet<string> GetVisitedUrls()
        {
            return visitedUrls;
        }

        public List<string> GetErrorUrls()
        {
            return errorUrls;
        }
    }

    public class CrawlerForm : Form
    {
        private Crawler crawler;
        private ListBox visitedListBox;
        private ListBox errorListBox;

        public CrawlerForm(Crawler crawler)
        {
            this.crawler = crawler;
            InitializeUI();
        }

        private void InitializeUI()
        {
            this.Text = "Web Crawler";
            this.Size = new System.Drawing.Size(600, 400);

            visitedListBox = new ListBox();
            visitedListBox.Dock = DockStyle.Left;
            visitedListBox.Width = this.Width / 2;
            this.Controls.Add(visitedListBox);

            errorListBox = new ListBox();
            errorListBox.Dock = DockStyle.Right;
            errorListBox.Width = this.Width / 2;
            this.Controls.Add(errorListBox);

            UpdateLists();
        }

        private void UpdateLists()
        {
            visitedListBox.Items.Clear();
            visitedListBox.Items.AddRange(crawler.GetVisitedUrls().ToArray());

            errorListBox.Items.Clear();
            errorListBox.Items.AddRange(crawler.GetErrorUrls().ToArray());
        }
    }

}
